{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torchvision import datasets, transforms\n",
    "import torch.utils.data\n",
    "import numpy as np\n",
    "import math\n",
    "from copy import deepcopy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_var(x, requires_grad=False):\n",
    "    \"\"\"\n",
    "    Automatically choose cpu or cuda\n",
    "    \"\"\"\n",
    "    if torch.cuda.is_available():\n",
    "        x = x.cuda()\n",
    "    return x.clone().detach().requires_grad_(requires_grad)\n",
    "\n",
    "\n",
    "class MaskedLinear(nn.Linear):\n",
    "    def __init__(self, in_features, out_features, bias=True):\n",
    "        super(MaskedLinear, self).__init__(in_features, out_features, bias)\n",
    "        self.mask_flag = False\n",
    "        self.mask = None\n",
    "\n",
    "    def set_mask(self, mask):\n",
    "        self.mask = to_var(mask, requires_grad=False)\n",
    "        self.weight.data = self.weight.data * self.mask.data\n",
    "        self.mask_flag = True\n",
    "\n",
    "    def get_mask(self):\n",
    "        print(self.mask_flag)\n",
    "        return self.mask\n",
    "\n",
    "    def forward(self, x):\n",
    "        if self.mask_flag:\n",
    "            weight = self.weight * self.mask\n",
    "            return F.linear(x, weight, self.bias)\n",
    "        else:\n",
    "            return F.linear(x, self.weight, self.bias)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLP(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MLP, self).__init__()\n",
    "        self.linear1 = MaskedLinear(28 * 28, 200)\n",
    "        self.relu1 = nn.ReLU(inplace=True)\n",
    "        self.linear2 = MaskedLinear(200, 200)\n",
    "        self.relu2 = nn.ReLU(inplace=True)\n",
    "        self.linear3 = MaskedLinear(200, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = x.view(x.size(0), -1)\n",
    "        out = self.relu1(self.linear1(out))\n",
    "        out = self.relu2(self.linear2(out))\n",
    "        out = self.linear3(out)\n",
    "        return out\n",
    "\n",
    "    def set_masks(self, masks):\n",
    "        # Should be a less manual way to set masks\n",
    "        # Leave it for the future\n",
    "        self.linear1.set_mask(masks[0])\n",
    "        self.linear2.set_mask(masks[1])\n",
    "        self.linear3.set_mask(masks[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, device, train_loader, optimizer, epoch):\n",
    "    model.train()\n",
    "    total = 0\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        data, target = data.to(device), target.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = F.cross_entropy(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        total += len(data)\n",
    "        progress = math.ceil(batch_idx / len(train_loader) * 50)\n",
    "        print(\"\\rTrain epoch %d: %d/%d, [%-51s] %d%%\" %\n",
    "              (epoch, total, len(train_loader.dataset),\n",
    "               '-' * progress + '>', progress * 2), end='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(model, device, test_loader):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    with torch.no_grad():\n",
    "        for data, target in test_loader:\n",
    "            data, target = data.to(device), target.to(device)\n",
    "            output = model(data)\n",
    "            test_loss += F.cross_entropy(output, target, reduction='sum').item()  # sum up batch loss\n",
    "            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability\n",
    "            correct += pred.eq(target.view_as(pred)).sum().item()\n",
    "\n",
    "    test_loss /= len(test_loader.dataset)\n",
    "\n",
    "    print('\\nTest: average loss: {:.4f}, accuracy: {}/{} ({:.0f}%)'.format(\n",
    "        test_loss, correct, len(test_loader.dataset),\n",
    "        100. * correct / len(test_loader.dataset)))\n",
    "    return test_loss, correct / len(test_loader.dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def weight_prune(model, pruning_perc):\n",
    "    '''\n",
    "    Prune pruning_perc % weights layer-wise\n",
    "    '''\n",
    "    threshold_list = []\n",
    "    for p in model.parameters():\n",
    "        if len(p.data.size()) != 1: # bias\n",
    "            weight = p.cpu().data.abs().numpy().flatten()\n",
    "            threshold = np.percentile(weight, pruning_perc)\n",
    "            threshold_list.append(threshold)\n",
    "\n",
    "    # generate mask\n",
    "    masks = []\n",
    "    idx = 0\n",
    "    for p in model.parameters():\n",
    "        if len(p.data.size()) != 1:\n",
    "            pruned_inds = p.data.abs() > threshold_list[idx]\n",
    "            masks.append(pruned_inds.float())\n",
    "            idx += 1\n",
    "    return masks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main():\n",
    "    epochs = 2\n",
    "    batch_size = 64\n",
    "    torch.manual_seed(0)\n",
    "\n",
    "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "    train_loader = torch.utils.data.DataLoader(\n",
    "        datasets.MNIST('../data/MNIST', train=True, download=False,\n",
    "                       transform=transforms.Compose([\n",
    "                           transforms.ToTensor(),\n",
    "                           transforms.Normalize((0.1307,), (0.3081,))\n",
    "                       ])),\n",
    "        batch_size=batch_size, shuffle=True)\n",
    "    test_loader = torch.utils.data.DataLoader(\n",
    "        datasets.MNIST('../data/MNIST', train=False, download=False, transform=transforms.Compose([\n",
    "            transforms.ToTensor(),\n",
    "            transforms.Normalize((0.1307,), (0.3081,))\n",
    "        ])),\n",
    "        batch_size=1000, shuffle=True)\n",
    "\n",
    "    model = MLP().to(device)\n",
    "    optimizer = torch.optim.Adadelta(model.parameters())\n",
    "    \n",
    "    for epoch in range(1, epochs + 1):\n",
    "        train(model, device, train_loader, optimizer, epoch)\n",
    "        _, acc = test(model, device, test_loader)\n",
    "    \n",
    "    print(\"\\n=====Pruning 60%=======\\n\")\n",
    "    pruned_model = deepcopy(model)\n",
    "    mask = weight_prune(pruned_model, 60)\n",
    "    pruned_model.set_masks(mask)\n",
    "    test(pruned_model, device, test_loader)\n",
    "        \n",
    "    return model, pruned_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train epoch 1: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.1098, accuracy: 9655/10000 (97%)\n",
      "Train epoch 2: 60000/60000, [-------------------------------------------------->] 100%\n",
      "Test: average loss: 0.0898, accuracy: 9736/10000 (97%)\n",
      "\n",
      "=====Pruning 60%=======\n",
      "\n",
      "\n",
      "Test: average loss: 0.0840, accuracy: 9731/10000 (97%)\n"
     ]
    }
   ],
   "source": [
    "model, pruned_model = main()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 可视化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_weights(model):\n",
    "    modules = [module for module in model.modules()]\n",
    "    num_sub_plot = 0\n",
    "    for i, layer in enumerate(modules):\n",
    "        if hasattr(layer, 'weight'):\n",
    "            plt.subplot(131+num_sub_plot)\n",
    "            w = layer.weight.data\n",
    "            w_one_dim = w.cpu().numpy().flatten()\n",
    "            plt.hist(w_one_dim[w_one_dim!=0], bins=50)\n",
    "            num_sub_plot += 1\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3df5TU9X3v8edLiOaXVAlCV9Au9mxyUWJW5QipqVXJIpIeaTRJJTZi4ITE6L2JmnOzNjlXY2oPaWv8cWOtplLRm4BGQ+VEotmg3Nx4RMC4UZEQFqQB3QIWNBhbFfO+f3w/g7PszDK7Mzs/dl+Pc+bMzPv7/c5+dj+z857v5/v5oYjAzMyGt0NqXQAzM6s9JwMzM3MyMDMzJwMzM8PJwMzMgJG1LsBAjRkzJpqbm2tdjGHvySeffCkijqrU67le64PrdegqVrcNmwyam5tZt25drYsx7En6t0q+nuu1Prheh65idetmIjMrStLlktZLelbSEknvlDRR0hOSNkm6R9KhtS6nlc/JwMwKkjQe+B/AlIiYDIwALgC+BdwQES3AHmB+7UppleJkYGZ9GQm8S9JI4N1AN3AWcF/avhj4ixqVzSrIycDMCoqIF4B/AH5DlgReAZ4EXo6IfWm37cD4QsdLWiBpnaR1u3btqkaRrQxOBmZWkKQjgdnAROBo4D3AOQV2LTjBWUTcHhFTImLKUUdVrGOSDRInAzMr5qPA8xGxKyLeBH4I/AlwRGo2ApgAvFirAlrlOBlYD9u2bePMM89k0qRJnHDCCdx0000AXHPNNYwfP57W1lZaW1tZsWLF/mMkXSWpS9JGSWfnxWemWJek9ur/Nlam3wDTJL1bkoDpwHPAo8An0j5zgQdqVD6roIYdZ2CDY+TIkVx//fWcfPLJ7N27l1NOOYW2tjYALr/8cr7yla8ceMg7yXqYnEDWlPBTSe9P224B2sjalddKWh4Rz1XlF7GyRcQTku4DfgHsA54CbgceBJZK+psUu6N2pbRKcTKwHpqammhqagLg8MMPZ9KkSbzwwgt9HXIEcEtEvA48L6kLODVt64qILQCSlpK1PzsZNJCIuBq4+oDwFt6uYxsi3ExUp5rbH+xxq4WtW7fy1FNPMXXqVAC+853vcOKJJzJv3jz27NmT2+1QYFveYbneJeOLxHsZar1OallnVn1Dpb6dDKygV199lfPPP58bb7yRUaNGcckll7B582Y6Oztpamriyiuv7OvwAFQk3jvoXidmNedkYL28+eabnH/++Vx44YWcd955AIwbN44RI0ZwyCGH8LnPfY41a9bkdn8DOCbv8Fzvku1F4mZWRC3PMpwMrIeIYP78+UyaNIkrrrhif7y7u3v/42XLljF58uTc05eBCyQdJmki0AKsAdYCLWkem0PJLjIvr9KvYWb95AvI1sNjjz3G3XffzQc/+EFaW1sB+Nu//VuWLFlCZ2cnkmhubua2227LHfJfwDKyC8P7gEsj4i0ASZcBD5PNabMoItZX+/cxs9I4GVgPH/nIR4jo3bQ/a9asosdExHXAdQXiK4AVvY8ws3rjZiIzM3MyMBssQ6XLoQ0PbiYyqxB/8FsjO+iZgaRjJD0qaUNa8ehLKT5aUkda7agjzXCIMjen+WielnRy3mvNTftvkjQ3L36KpGfSMTeneVDMzKxKSmkm2gdcGRGTgGnApZKOB9qBlWm1o5XpOWRT3Lak2wLgVsiSB9mw9qlkQ9mvziWQtM+CvONmlv+rmZmVZzg19R00GUREd0T8Ij3eC2wgm1ZgNtkqR9BztaPZwF2RWU023W0TcDbQERG7I2IP0AHMTNtGRcTjkXVjuQuvnNTLcHpTDjWuO2sE/bqALKkZOAl4AhgXEd2QJQxgbNqt2Jw0fcW3F4ibmVmVlJwMJL0XuB/4ckT8tq9dC8T6mqum5DlshtqEZmZm9aKkZCDpHWSJ4HsR8cMU3pGaeEj3O1O82Jw0fcUnFIj34gnNrF65GcgaXSm9iUS2eMWGiPh23qblZKscQc/VjpYDF6VeRdOAV1Iz0sPADElHpgvHM4CH07a9kqaln3URw3jlJLcvm1ktlDLO4DTgM8AzkjpT7K+BhcC9kuaTLY/3ybRtBTAL6AJeAz4LEBG7JX2TbAIzgGsjYnd6fAlwJ/Au4MfpZmY25OW+/G1d+LGaluOgySAifk7hdn3I1kQ9cP8ALi3yWouARQXi64DJvY8wq28+i7OhwiOQzQZgOCQBSR8A7skLHQf8L7Lu3/cAzcBW4FOpu7g1MM9NZGYFRcTGiGiNiFbgFLJm32UUH3BqDczJwMxKMR3YHBH/RvEBp9bAnAzMrBQXAEvS42IDTnvwuKDG4mRgZn1Ky5aeC/ygP8d5XFBjcTIws4M5B/hFROxIz4sNOLUG5mRgZgczh7ebiKD4gFMrQ617qDkZmFVRo40wl/RuoA34YV54IdAmaVPatrAWZbPK8jgDMysqIl4D3ndA7D8oMOB0uCk3qdfblwKfGZiZmZOBmZm5mcisX+rt1N4qq1KTxtXL5HP94TMDMzNzMjCrFp9VWD1zMjAzMycDMzMrbdnLRZJ2Sno2L3aPpM5025pbAU1Ss6T/zNv2T3nHnCLpGUldkm5OS1wiabSkDkmb0v2Rg/GLmplZcaWcGdwJzMwPRMRf5s1zfj89Ryduzm2LiC/kxW8FFgAt6ZZ7Tc+NbmZWYwdNBhHxM2B3oW3p2/2n6DlvSaH9moBREfF4WhbzLt6eA91zo5uZ1Vi54wz+FNgREZvyYhMlPQX8Fvh6RPw/YDywPW+f7SkGB8yNLqng3OiQzY9OdnbBscceW2bRzcyqr157lZV7AfnA2Qy7gWMj4iTgCuD7kkYBKnBs9PeHeX50M7PBMeBkIGkkcB55C2ZHxOtpEisi4klgM/B+sjOBCXmHTwBeTI89N3qd2bZtG2eeeSaTJk3ihBNO4KabbgJg9+7dtLW10dLSQltbG3v2ZGugK3Nz6hzwtKSTc68laW7qHLBJ0tzCP9HMaq2cM4OPAr+KiP3NP5KOkjQiPT6O7ELxltQMtFfStHSd4SLengPdc6PXmZEjR3L99dezYcMGVq9ezS233MJzzz3HwoULmT59Ops2bWL69OksXLh/5uJzeLtjwAKyzgJIGg1cDUwFTgWudm8xG04qNWV5NaY+P+g1A0lLgDOAMZK2A1dHxB30XBM153TgWkn7gLeAL0RE7uLzJWQ9k94F/DjdIJsL/V5J84HfAJ8s5xey8jU1NdHU1ATA4YcfzqRJk3jhhRd44IEHWLVqFQBz587ljDPOyB0yG7grdQ5YLemIdJZ3BtCRew9I6iDrRdZnhwMzq76DJoOImFMkfnGB2P1kXU0L7b8OmFwg7rnR69jWrVt56qmnmDp1Kjt27NifJJqamti5c3+L3nhgW95huQ4CxeI9uGOA1VqlvnXX68XhUngEshX16quvcv7553PjjTcyatSovnYt1kGgpI4D7hhgVntOBlbQm2++yfnnn8+FF17IeeedB8C4cePo7u4GoLu7m7Fj9/cC3g4ck3d4roNAsbg1iNTkd5+kX0naIOnDnjVgaHIysF4igvnz5zNp0iSuuOKK/fFzzz2XxYuz8YGLFy9m9uzZuU3LgYtSr6JpwCup08DDwAxJR6YPjBkpZo3jJuChiPhvwIeADXjWgCHJyaDBVKNN8rHHHuPuu+/mkUceobW1ldbWVlasWEF7ezsdHR20tLTQ0dFBe/v+z4AVwBagC/gu8EWAdOH4m8DadLs2r0OB1bk0Ruh04A6AiHgjIl7GswYMSV7pzHr5yEc+QtYxqLeVK1f2iqVeRJcW2j8iFgGLKlk+q5rjgF3Av0j6EPAk8CX6MWuANQ6fGZhZMSOBk4Fb06wCv6MfTUKSFkhaJ2ndrl27BquMDaWeexs5GZhZMduB7RHxRHp+H1lyKGnWAPcSayxuJjIrQT1/oxssEfHvkrZJ+kBEbCQbD/Rcus0lGzDasLMG9KdOm9sfbKjF7QfCycDM+vLfge9JOpSsk8BnyVoUPGvAEONkYGZFRUQnMKXAJs8aMMT4moGZmTkZmJmZk4GZWUU0eicDJwMzM3MyMDOzEpKBpEWSdkp6Ni92jaQXJHWm26y8bVel5Q83Sjo7Lz4zxboktefFJ0p6Is2AeE/qwmZmVlfKWW2sv8dWY2WzA5VyZnAn2epUB7ohIlrTbQWApOPJVkA7IR3zj5JGpKUwbyFbHvF4YE7aF+Bb6bVagD3A/HJ+ITMz67+DJoOI+BlQ6kyTs4GlEfF6RDxPNovlqenWFRFbIuINYCkwO62HfBbZMHfwDIhmZjVRzqCzyyRdBKwDroyIPWRLGq7O2yd/mcMDlz+cCrwPeDki9hXYvxcvj2hmtdbovYaKGegF5FuBPwZagW7g+hTv7/KHJS2LuH+DJ74yMxsUAzoziIgduceSvgv8KD3ta5nDQvGXgCMkjUxnB14W0YaF3LfLoT75mTWOAZ0Z5KavTT4O5HoaLQcukHSYpIlAC7CGbJWrltRz6FCyi8zL06IojwKfSMc37AyIZmaN7KBnBpKWAGcAYyRtB64GzpDUStaksxX4PEBErJd0L9kUt/uASyPirfQ6l5GtfzsCWBQR69OP+CqwVNLfAE+RltgzM7PqOWgyiIg5BcJFP7Aj4jrgugLxFWRr5R4Y30LW28jMzGrEI5DNzMzJwMzMnAzMbIirxdQOjcjJwMzMvOylmRUnaSuwF3gL2BcRUySNBu4Bmsl6E34qzUBgDcxnBmYH4SYGzkwTUubWQm4HVqbJJVem59bgnAzMrL9mk00qCZ5ccshwMjCzvgTwE0lPpokiAcZFRDdAuh9b6EBJCyStk7Ru165dVSquDZSvGZhZX06LiBcljQU6JP2q1AMj4nbgdoApU6YUnYDS6oPPDMysqIh4Md3vBJaRzRawIzc/WbrfWbsSWqU4GdQRX6i0eiLpPZIOzz0GZpBNSrmcbFJJ8OSSQ4abicysmHHAsmxBQkYC34+IhyStBe6VNB/4DfDJGpbRKsTJwKyG6nldgzSJ5IcKxP8DmF79ElWGz8ALczOR9TJv3jzGjh3L5MmT98euueYaxo8fT2trK62traxY8fYEtJKuktQlaaOks/PiM1OsS5L7opvVMScD6+Xiiy/moYce6hW//PLL6ezspLOzk1mzZuXC7yRbrOgEYCbwj5JGSBoB3AKcAxwPzJF0fFV+ATPrt4MmA0mLJO2U9Gxe7O8l/UrS05KWSToixZsl/aekznT7p7xjTpH0TPqWeLNSQ6Sk0ZI6JG1K90cOxi9qpTv99NMZPXp0qbsfASyNiNcj4nmgi6zHyalAV0RsiYg3gKVkg5XMasIT1vWtlDODO8m+8eXrACZHxInAr4Gr8rZtTkPXWyPiC3nxW4EFZEthtuS9poe2N4jvfOc7nHjiicybN489e/ZPRXMosC1vt+3A+HQrFO/Fg5PMau+gySAifgbsPiD2k7SAPcBqsoXsi0p9kUdFxONp3eO7eHsIu4e2N4BLLrmEzZs309nZSVNTE1deeWVfuwegIvHewYjbI2JKREw56qijKlFcM+unSlwzmAf8OO/5RElPSfq/kv40xcaTfTPMyf+WWNLQdvA3yFoaN24cI0aM4JBDDuFzn/sca9asyW16Azgmb9cJwItkdVwobmYDNJjNXGUlA0lfI1v4/nsp1A0cGxEnAVcA35c0in58S+yLv0HWTnd39/7Hy5Yty+9p9DJwgaTDJE0kawJcA6wFWiRNlHQo2UXm5dUttZmVasDjDCTNBf4cmJ6afoiI14HX0+MnJW0G3k/2LTG/KSn/W+IOSU0R0e2h7fVhzpw5rFq1ipdeeokJEybwjW98g1WrVtHZ2Ykkmpubue2223K7/xfZNAXPkX0xuDQi3gKQdBnwMDACWBQR62vw65hZCQaUDCTNBL4K/FlEvJYXPwrYHRFvSTqO7FvilojYLWmvpGnAE8BFwP9Oh+WGti/EQ9vrwpIlS3rF5s+fX3T/iLgOuK5AfAWwovcRZlZvDpoMJC0BzgDGSNoOXE3We+gwslkMAVannkOnA9dK2ke2MtIXIiJ38fkSsp5J7yK7xpC7zrAQD203M6upgyaDiJhTIHxHkX3vB+4vsm0dMLlAvKGHtpvZ0DMcxyN4BLKZmTkZmJmZZy01K2o4NhVYfanme9BnBmZm5mRgZmZOBmZmhpOBmR1EWp/iKUk/Ss8nSnoiTTt/T5puxBqck4GZHcyXgA15z78F3JCmnd8DFB+ebg3DycDMipI0AfgY8M/puYCzgPvSLp52fohwMjCzvtwI/E/g9+n5+4CX89YzKbpokTUWJwMzK0jSnwM7I+LJ/HCBXQtOR+/1RxqLk4GZFXMacK6krWRrWJ9FdqZwhKTcgNWiixZ5/ZHG4mRgZgVFxFURMSEimskWJ3okIi4EHgU+kXar62nnPYq8dE4GZnWguf3BRvrg+ipwhaQusmsIBWcxtsbiuYnM7KAiYhWwKj3eApxay/JY5fnMwMzMSksGkhZJ2inp2bzYaEkdaRRih6QjU1ySbpbUJelpSSfnHTM37b8praGci58i6Zl0zM2pL7OZmVVJqWcGdwIzD4i1AyvTKMSV6TnAOWRrH7cAC4BbIUseZEtmTiU7xbw6l0DSPgvyjjvwZ5mZ2SAqKRlExM+A3QeEZ5ONPoSeoxBnA3dFZjVZN7Qm4GygIyJ2R8QeoAOYmbaNiojHIyKAu/CIRjOzqirnmsG4iOgGSPdjU3w8sC1vv9wIxb7i2wvEe/EgFjOzwTEYF5CLjVDsb7x30INYzMwGRTnJYEdq4iHd70zx7cAxefvlRij2FZ9QIG5mZlVSTjJYTjb6EHqOQlwOXJR6FU0DXknNSA8DMyQdmS4czwAeTtv2SpqWehFdRB2PaDQzG4pKGnQmaQlwBjBG0nayXkELgXslzQd+A3wy7b4CmAV0Aa8BnwWIiN2SvgmsTftdGxG5i9KXkPVYehfw43QzM7MqKSkZRMScIpumF9g3gEuLvM4iYFGB+DpgcillMTOzyvMIZDMz89xEjSg3odnWhR+rcUmGpgaaMM6sYnxmYGZmTgZmZuZkYGZmOBlYAfPmzWPs2LFMnvx2B6/du3fT1tZGS0sLbW1t7NmzZ/+2/s5Sa2b1x8nAern44ot56KGHesQWLlzI9OnT2bRpE9OnT2fhwoW5TX9A/2epNbM642RgvZx++umMHj26R+yBBx5g7tzsy/3cuXP513/919ymI+jHLLXV+h3MrH+cDKwkO3bsoKmpCYCmpiZ27sxNRcU76N8stb14Ntr6JOmdktZI+qWk9ZK+keITJT2Rmv/ukXRorctq5XMysMHg2WgHqM7GOLwOnBURHwJaydYfmQZ8C7ghLWy1B5hfwzJahTgZWEnGjRtHd3c3AN3d3Ywdm1u+gjfp3yy11iBS09+r6ek70i2As4D7Ujx/YStrYE4GVpJzzz2XxYuzhe0WL17M7Nmzc5teph+z1Nag6FYGSSMkdZJNUd8BbAZejoh9aRc3/w0RTgbWy5w5c/jwhz/Mxo0bmTBhAnfccQft7e10dHTQ0tJCR0cH7e25Ja95BdhCNkvtd4EvQjZLLZCbpXYtPWeptQYREW9FRCvZmd2pwKRCuxU51s1/DcRzE1kvS5YsKRhfuXJlwXhE9GuWWms8EfGypFXANLIeYyPT2YGb/4YInxmYWUGSjpJ0RHr8LuCjwAbgUeATabf8ha1qrrn9wXq7CN8wBpwMJH1AUmfe7beSvizpGkkv5MVn5R1zVRqpulHS2XnxmSnWJam98E80syprAh6V9DRZU19HRPwI+CpwhaQu4H3AHTUso1XIgJuJImIjWXczJI0AXgCWka1sdkNE/EP+/pKOBy4ATgCOBn4q6f1p8y1AG9nFqLWSlkfEcwMtm5mVLyKeBk4qEN9Cdv3AhpBKXTOYDmyOiH/LljEuaDawNCJeB55P3ypyb6iu9AZD0tK0r5OBmVmVVOqawQVA/lXHy9KkZYvy5qMpNiLVI1XNzGqs7GSQhqKfC/wghW4F/pisCakbuD63a4HDPVLVzKwfBusieSWaic4BfhEROwBy9wCSvgv8KD3ta0TqsB6p6t4PZlZrlWgmmkNeE1GasTLn48Cz6fFy4AJJh0maSDbl8RqyXgotafKrQ8manJZXoFxmZlaiss4MJL2brBfQ5/PCfyeplaypZ2tuW0Ssl3Qv2YXhfcClEfFWep3LyKYqGAEsioj15ZTLzIY3n233X1nJICJeI+tnnB/7TB/7XwdcVyC+AlhRTlnMzGzgPALZzKwBVfpCsucmMkvctGDDmc8MzMzMycDMzJwMzMwMJwOzuuNpmK0WnAzMzMzJwMzMnAzMzAwnAzMzw8nAzMzwCGQzK0LSMcBdwB8Cvwduj4ibJI0G7gGaySaj/FRE7KlVOd3zqjJ8ZmBmxewDroyIScA04NK0lnk7sDIiWoCV6bk1OCcDMysoIroj4hfp8V5gA9mStLOBxWm3xcBf1KaEVklOBmZ2UJKagZOAJ4BxEdENWcIAxhY5xmuWN5BKrIG8VdIzkjolrUux0ZI6JG1K90emuCTdLKlL0tOSTs57nblp/02S5pZbLrNGVy8jkSW9F7gf+HJE/LbU47xmeWOp1JnBmRHRGhFT0vNibYrnkC132QIsAG6FLHkAVwNTgVOBq3MJxKwa6uFDtx5JegdZIvheRPwwhXfklrdN9ztrVT6rnMFqJirWpjgbuCsyq4Ej0pvpbKAjInanXgkdwMxBKpuZlUCSgDuADRHx7bxNy4Hc2ftc4IFql80qrxLJIICfSHpS0oIUK9amOB7Ylnfs9hQrFu/BbZBmVXUa8BngrNQM3ClpFrAQaJO0iWwN9IW1LKRVRiXGGZwWES9KGgt0SPpVH/uqQCz6iPcMRNwO3A4wZcqUXtsbjZsmrJ5FxM8p/L8JML2aZbHBV/aZQUS8mO53AsvI2vyLtSluB47JO3wC8GIfcTMzq4KykoGk90g6PPcYmAE8S/E2xeXARalX0TTgldSM9DAwQ9KR6cLxjBQzM7MqKLeZaBywLLvOxEjg+xHxkKS1wL2S5gO/AT6Z9l8BzAK6gNeAzwJExG5J3wTWpv2ujYjdZZbNBkFzczOHH344I0aMYOTI7O1TbHqCdAHyJrI6fw24ODeIyczqS1nJICK2AB8qEP8PCrQpRkQAlxZ5rUXAonLKY9Xx6KOPMmbMGADSF4FcV+KFktrT86/SsyvxVLKuxFNrUWYz65tHIDewehmURP+7EptZnfGspdYvkpgxYwaS+PznP58L9+hKnHqWQfEuw91VK7CZlcTJwPrlscce4+ijj2bnzp20tbUBvLeP3UvqMpzGpywAOPbYYytSTjPrHzcTWb8cffTRAIwdO5aPf/zjAO+h/12Je/AcNma152RgJfvd737H3r179z/+yU9+AvCf9L8rsZnVGTcTWcl27NiROxtg3759fPrTn+bxxx//Ldl0BCV3JTaz+uNkYCU77rjj+OUvf9kj9vWvf31AXYnNylUnPemGDDcTmZmZk4GZmTkZmNW9OhpcaEOYrxnUgP+x64frwizjMwMzM3MyMDMzJwMz64OkRZJ2Sno2LzZaUoekTen+yFqW0SrDycDM+nInMPOAWG7K8hZgZXpuDW7AyUDSMZIelbRB0npJX0rxayS9cMAC2rljrpLUJWmjpLPz4jNTrCvNh29mdSAifgYcuNBUsSnLrYGV05toH3BlRPwiLX35pKSOtO2GiPiH/J0lHQ9cAJwAHA38VNL70+ZbgDayic3WSloeEc+VUTYzGzzFpizvwbPRNpYBnxlERHduCcOI2AtsIJurvpjZwNKIeD0iniebr+bUdOuKiC0R8QawNO1rJXI/9OGh0erYs9E2lopcM5DUDJwEPJFCl0l6Ol18yl1cKrbQSbH4kOMPbRsiik1Zbg2s7GQg6b3A/cCXI+K3ZOvc/jHQSrai1fW5XQscHn3EC/2sBZLWSVq3a9eucotuZgNTbMpya2BlJQNJ7yBLBN+LiB8CRMSOiHgrIn4PfJesGQiKL3RS0gIo6bV92mlWRZKWAI8DH5C0PU1TvhBok7SJ7FrfwlqW0SpjwBeQJQm4A9gQEd/OizflLWDycSDXP3k58H1J3ya7gNwCrCE7M2iRNBF4gewi86cHWi4zq5yImFNkU68py62xldOb6DTgM8AzkjpT7K+BOZJayZp6tgKfB4iI9ZLuBZ4j64l0aUS8BSDpMuBhYASwKCLWl1EuMzPrpwEng4j4OYXb+1f0ccx1wHUF4iv6Os6s0nwh36wnj0A2ayDDuUfacP7dq8HJwMzMnAyGEn9zMrOBcjKoEn9Im/Wf/2+qxyud2bDiDxezwnxmYGZmPjMwa0S5M5ytCz9W45IMDp/BVZ/PDMzMzGcGg60W33Ca2x8cst8YzXzW0FOlzhKdDGxY8AeIWd/cTGTWwDy2xCrFZwaDxP+g9cN1Uf/6qiPXX2kK/Z3603TkM4Mhyt8Yzaw/nAzMrCb8ZaW+uJmowvwGt1oY6uMObPA5GVRQPSaC4fwhUY/1YVav6iYZSJoJ3ES22tk/R0TDrKvqD53iGrleG1H+e3GwvwBUom79v1M/6iIZSBoB3EK2uPZ2YK2k5RHxXG1L1rdGeiPX4gyh2vXaSPVRDYNZ5436P2vF1UUyAE4FuiJiC4CkpcBssvWSa2KofrCU2/2snwatXvM/6IZqXVXKICWFuvuftfLUSzIYD2zLe74dmHrgTpIWAAvS01clbRyEsowBXhqE1x2IqpRF3yppt2Jl+aM+jhn0es0rez3VG9RXecYAL5VYzzl91SuUULdV+n8tVb3UR1XLUaTOC9ZtvSQDFYhFr0DE7cDtg1oQaV1ETBnMn1GqIVCWqtVrPf2toL7KM0hlOWjdVuP/tVT1Uh/1Uo5C6mWcwXbgmLznE4AXa1QWqxzX69Dluh1i6iUZrAVaJE2UdChwAbC8xmWy8rlehy7X7RBTF81EEbFP0mXAw2Td1BZFxPoaFacuTmuThi5Lleu1nv5WUF/lqXhZ6ux/thT1Uh/1Uo5eFNGrCdfMzIaZemkmMjOzGnIyMDOz4ZkMJI2W1CFpU7o/ssA+rSrrFZQAAANvSURBVJIel7Re0tOS/jJv252SnpfUmW6tAyjDTEkbJXVJai+w/TBJ96TtT0hqztt2VYpvlHR2f392P8txhaTn0t9gpaQ/ytv2Vt7foKoXD8utwwqVYcB1OBjKqcuhptbvj3p7b5QkIobdDfg7oD09bge+VWCf9wMt6fHRQDdwRHp+J/CJMn7+CGAzcBxwKPBL4PgD9vki8E/p8QXAPenx8Wn/w4CJ6XVGDGI5zgTenR5fkitHev5qo9ZhBX7+gOtwkP4eZdXlULvV8v1Rb++NUm/D8syAbNj84vR4MfAXB+4QEb+OiE3p8YvATuCoCv38/UP5I+INIDeUv1gZ7wOmS1KKL42I1yPieaArvd6glCMiHo2I19LT1WT9yetBI9fhYGjkuhwMtXx/1Nt7oyTDNRmMi4hugHQ/tq+dJZ1KluE354WvS6eWN0g6rJ8/v9BQ/vHF9omIfcArwPtKPLaS5cg3H/hx3vN3SlonabWkXv9sg6wSdViOcupwMJRbl0NNLd8f9fbeKEldjDMYDJJ+CvxhgU1f6+frNAF3A3Mj4vcpfBXw72RvntuBrwLX9udlC8QO7ONbbJ+SpnioYDmyHaW/AqYAf5YXPjYiXpR0HPCIpGciolIftoNdh+Uqpw4HQ7l12XDq+P1Rb++NkgzZZBARHy22TdIOSU0R0Z3eCDuL7DcKeBD4ekSsznvt7vTwdUn/Anyln8UrZSh/bp/tkkYCfwDsLvHYSpYDSR8l+wf7s4h4PRdPp9ZExBZJq4CTqNw370Gtwwoopw4HQ1l12Yjq+P1Rb++NkgzXZqLlwNz0eC7wwIE7KBtivwy4KyJ+cMC2pnQvsrbIZ/v580sZyp9fxk8Aj0R2tWk5cEHqjTARaAHW9PPnl1wOSScBtwHnRsTOvPiRueYxSWOA06ju9MVl1WEFlFOHg2HAdTlE1fL9UW/vjdLU+gp2LW5kbXMrgU3pfnSKTyFbsQngr4A3gc68W2va9gjwDFkS+D/AewdQhlnAr8m+SX8txa4l+0cFeCfwA7ILxGuA4/KO/Vo6biNwTpl/i4OV46fAjry/wfIU/5P0N/hlup/fSHVYoTIMuA4H6W8yoLocirdavz/q7b1Rys3TUZiZ2bBtJjIzszxOBmZm5mRgZmZOBmZmhpOBmZnhZGBmZjgZmJkZ8P8B2+cFJVEPk9sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 3 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_weights(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dfZScZX3/8ffHrGBRIETYNGRJN9jwmOIathBqm4ppQkAPEUXNipKYnEYp+GuFtoSfnoNocwxWi3BUaiSR4M8mRAqaIxAMkdTWEx6CLOGpaR6gsBITIAkGaYHA9/fHfU2YbGY287Q7M7uf1zlzZuZ7X/fstXvPznfu674eFBGYmdnQ9pZ6V8DMzOrPycDMzJwMzMzMycDMzHAyMDMzoKXeFajUkUceGe3t7fWuxpD34IMPPh8RR9Xq9XxcG4OP6+BV7Ng2bTJob29n3bp19a7GkCfpv2v5ej6ujcHHdfAqdmzdTGRmZk4GZmbmZGBmZjgZWAGzZ8+mtbWV8ePH7419/OMfp6Ojg46ODtrb2+no6MhtOkjS/0jqTrd/zm2QdKqkRyRtknSdJA3wr2JmJWraC8jWf2bNmsUll1zChRdeuDd2880373182WWXcfjhh+fvsjkiOtjf9cBc4F7gDmAacGd/1NnMquMzA9vPpEmTGDFiRMFtEcHy5cvp6urq8zUkjQIOi4i1kc2GeBPwoZpX1sxqwsnAyvLv//7vjBw5knHjxuWHx0p6SNK/SfqzFBsN9OSV6UkxM2tAbiaysixdurT3WcFrwJiIeEHSqcCPJZ0MFLo+UHC+dElzyZqTGDNmTI1rbGalcDKwku3Zs4dbb72VBx98MD8cEfFCevCgpM3AcWRnAm155dqAZwu9bkQsBBYCdHZ2eoENszpwM1ETaZ93O+3zbq/bz7/77rs54YQTaGvL/4ynRdIwAEnHAuOALRGxFdgtaWLqRXQh8JMBr3SN1Ptvb/UxlI65k4Htp6urizPOOIMNGzbQ1tbGokWLAFi2bFmhC8fvANZLehi4BfhsROxI2y4CbgA2AZtxTyKzhuVmItvP0qVLC8ZvvPHGQuFdEdFZaENErAPGF9pmZo3lgGcGkhZL2i7p0V7xz0naIOkxSV/Li1+RBhltkHRWXnxaim2SNC8vPlbSfZI2SrpZ0kG1+uXMzKw0pTQT3Ug2WGgvSWcC04FTIuJk4OspfhIwAzg57fMdScNSm/K3gbOBk4CuVBbgauCaiBgH7ATmVPtLmZlZeQ6YDCLiF8COXuGLgAUR8Uoqsz3FpwPLIuKViHiSrK34tHTbFBFbIuJVYBkwPV1YfD9ZWzPAEjwwycxswFV6Afk44M9S886/SfrjFB8NPJNXLjfQqFj8nWRtznt6xc3MbABVegG5BTgCmAj8MbA8dSssNtCoUNKJPsoX5MFJZmb9o9Izgx7g1sjcD7wBHJnix+SVyw00KhZ/HhguqaVXvKCIWBgRnRHRedRRNVuRz8wKkHR83my03ZJ+K+lvJI2QtCp1+lgl6Yh619WqV2ky+DFZWz+SjgMOIvtgXwHMkHSwpLFkA5DuBx4AxqWeQweRXWRekSYwuwc4P73uTJp4YJLZYBIRGyKiI81IeyrwMnAbMA9YnTp9rE7PrcmV0rV0KbAWOF5Sj6Q5wGLg2NTddBkwM50lPAYsBx4HVgIXR8Tr6ZrAJcBdwBPA8lQW4HLgUkmbyK4hLKrtr2hmNTCZbKry/ybrKLIkxd3pY5A44DWDiCg2V/Eni5SfD8wvEL+DbE773vEtZL2NzBrSUJqSoA8zgNxoxJFpuhEiYquk1kI7+Bpfc/F0FGbWp9S0ey7wo3L28zW+5uJkYGYHcjbwq4jYlp5vS4sX5RYx2l50T2saTgZmdiBdvNlEBFlHkZnpsTt9DBJOBmZWlKRDgCnArXnhBcAUSRvTtgX1qJvVlmctNbOiIuJlsl5++bEXyHoX2SDiMwMzM3MyMDMzJwMzs72G8vKmTgZmZuZkYGZmTgZmdTWUmyWssTgZmJmZk4Htb/bs2bS2tjJ+/Pi9sS996UuMHj2ajo4OOjo6uOOON+cclHSFpE2SNkg6Ky8+LcU2SfI0x2YNzMnA9jNr1ixWrly5X/zzn/883d3ddHd3c8455+TCbyOb0fJkYBrwHUnDJA0Dvk02r81JQJekkwbkFzCzsjkZ2H4mTZrEiBEjSi0+HFgWEa9ExJPAJrIpyU8DNkXEloh4lWzdi+n9UmEzq5qTgZXsW9/6FqeccgqzZ89m586dufBBwDN5xXqA0elWKL4fSXMlrZO07rnnnuuHmpvZgZSy0tliSdvTqma9t/2tpJB0ZHouSdelNuL1kibklZ2Z1kzdKGlmXvxUSY+kfa6TpFr9clY7F110EZs3b6a7u5tRo0Zx2WWX9VU8gELHMQoW9rz3ZnVXypnBjWRtwfuQdAzZjIVP54XPJlv3eBzZCkfXp7IjgCuB08maD67MW0T7+lQ2t99+P8vqb+TIkQwbNoy3vOUt/OVf/iX3339/btOrwDF5RduAZ8nOBArFzawBHTAZRMQvgB0FNl0D/D37ftubDtyU1kO+FxieFr84C1gVETsiYiewCpiWth0WEWsjIoCb8HqqDWnr1q17H9922235PY12ATMkHSxpLFlCvx94ABgnaWxaKWsG2Tz4ZtaAKprCWtK5wK8j4uFerTrF2on7ivcUiFsddXV1sWbNGp5//nna2tq46qqrWLNmDd3d3Uiivb2d7373u7ni/wvcBjwO7AEujojXASRdAtwFDAMWR8Rjdfh1zKwEZSeDtNjFF4CphTYXiPXVflxyu3L62V5gewAsXbp0v9icOXOKlo+I+cD8AvE7gDv238PMGk0lZwbvAsYCubOCNuBXkk6jeDtxD/C+XvE1Kd5WoHxBEbEQWAjQ2dlZNGmYmZXDU4JU0LU0Ih6JiNaIaI+IdrIP9AkR8RuyNuELU6+iicCLEbGVrKlgqqQj0oXjqcBdadtuSRNTL6IL8XqqZmYDrpSupUuBtcDxknokFW8vyJoEtpANPPoe8FcAEbED+ArZRcUHgC+nGMBFwA1pn83AnZX9KmZmVqkDNhNFRNcBtrfnPQ7g4iLlFgOLC8TXAeP338PM6k3ScLIva+PJrufNBjYANwPtwFPAx1IvQWtiHoFsZn25FlgZEScA7waeAOYBqyNiHLA6Pbcm52RgZgVJOgyYBCwCiIhXI2IX2XiiJanYEjw2aFBwMjCzYo4FngO+L+khSTdIejswMnX+IN23FtrZc041FycDMyumBZgAXB8R7wF+RxlNQp5zqrk4GZhZMT1AT0Tcl57fQpYctqWpZEj32+tUP6uhiqajMBsKhvpApIj4jaRnJB0fERuAyWTTjjwOzAQWpPumHBs01I9vb04GZtaXzwE/TJMNbgE+TdaisDyNOXoa+Ggd62c14mRgZkVFRDfQWWDT5IGui/UvXzMwMzMnAzMzczIwMzOcDMzMDCcDMzPDycDMzHAyMDMznAzMzAwnAzMzo7RlLxdL2i7p0bzYP0r6T0nrJd2WVkPKbbtC0iZJGySdlReflmKbJM3Li4+VdJ+kjZJuTsPerY5mz55Na2sr48e/uQDd3/3d33HCCSdwyimncN5557Fr167cpoMk/Y+k7nT759wGSadKeiQd8+vSOtdm1oBKOTO4EZjWK7YKGB8RpwD/BVwBIOkkYAZwctrnO5KGSRoGfBs4GzgJ6EplAa4GrkmrJu0E+lpj2QbArFmzWLly5T6xKVOm8Oijj7J+/XqOO+44vvrVr+Zv3hwRHen22bz49cBcYFy69X4fmVmDOGAyiIhfADt6xX4WEXvS03uBtvR4OrAsIl6JiCfJFrk/Ld02RcSWiHgVWAZMT98U3082NS541aSGMGnSJEaMGLFPbOrUqbS0ZFNZTZw4kZ6enj5fI01tfFhErE1rY9+Ej61Zw6rFNYPZwJ3p8WjgmbxtPSlWLP5OYFdeYsnFC/LKSY1h8eLFnH322fmhsWklrH+T9GcpNprseOYUPbY+rmb1V1UykPQFYA/ww1yoQLGoIF6QV06qv/nz59PS0sIFF1yQC70GjEkrYV0K/EtaO7fkY+vj6rn1rf4qnsJa0kzgg8Dk1AwA2be/Y/KKtQHPpseF4s8DwyW1pLOD/PLWYJYsWcJPf/pTVq9eTd614IiIF9KDByVtBo4jey+05e3uY2vWwCo6M5A0DbgcODciXs7btAKYIelgSWPJLhreDzwAjEs9hw4iu8i8IiWRe4Dz0/5Nu2rSYLdy5UquvvpqVqxYwSGHHJK/qSV1EEDSsWTHfEtaKH23pInp2tCF+NiaNaxSupYuBdYCx0vqSasbfQs4FFiV350wIh4DlpMti7cSuDgiXk/f+i8B7gKeAJanspAllUslbSK7hrCopr+hla2rq4szzjiDDRs20NbWxqJFi7jkkkvYvXs3U6ZMoaOjg89+dm+noXcA6yU9TNYR4LMRketwcBFwA1lHgs28eW3JbMC0z7vdzXAlOGAzUUR0FQgX/cCOiPnA/ALxO4A7CsS3kPU2sgaxdOnS/WJz5hTt8bsrIgqthEVErAPGF9pmZo3FI5DNzMxrIJtZcZKeAnYDrwN7IqJT0gjgZqAdeAr4WETsrFcdrTZ8ZmBmB3JmGl2eaw6cB6xOswasTs+tyTkZmFm5ppPNFgCeNWDQcDIws74E8DNJD0qam2IjU9dh0n1roR09sry5+JqBmfXlvRHxrKRWsq7k/1nqjhGxEFgI0NnZWXRmAWsMPjMws6Ii4tl0vx24jawb+LY0EWFuQsLt9auh1YqTgZkVJOntkg7NPQamAo+SzTQwMxXzrAGDhJuJzKyYkcBtaR6qFuBfImKlpAeA5Wk2gqeBj9axjlYjTgZmVlCaHeDdBeIvAJMHvka14akpCnMzkZmZORmYmZmbicxsiHDzUN+cDMx68YeGDUVuJjIzMycDMzMrbaWzxZK2S3o0LzZC0ipJG9P9ESkuSddJ2iRpvaQJefvMTOU3pvWTc/FTJT2S9rlOeYvrmpnZwCjlzOBGYFqvWLEpbM8mWwN3HDAXuB6y5AFcCZxONpz9ylwCSWXm5u3X+2eZmVk/O2AyiIhfADt6hYtNYTsduCky9wLD09wlZwGrImJHWgRjFTAtbTssItZGRAA34elwzcwGXKXXDIpNYTsaeCavXE+K9RXvKRAvyFPimpn1j1pfQC7U3h8VxAuKiIUR0RkRnUcddVSFVTQzs94qTQbFprDtAY7JK9cGPHuAeFuBuNXR7NmzaW1tZfz48XtjO3bsYMqUKYwbN44pU6awc+ebS96W22nAsrEMHs/QuIbi8ak0GRSbwnYFcGHqVTQReDE1I90FTJV0RLpwPBW4K23bLWli6kV0IZ4Ot+5mzZrFypUr94ktWLCAyZMns3HjRiZPnsyCBQtymw6n/E4DZtZgSulauhRYCxwvqSdNW7sAmCJpIzAlPQe4A9gCbAK+B/wVQETsAL4CPJBuX04xgIuAG9I+m4E7a/OrWaUmTZrEiBEj9on95Cc/YebMLP/PnDmTH//4x7lNwymj08BA/Q5mVp4DTkcREV1FNu03hW3qEXRxkddZDCwuEF8HjN9/D2sk27ZtY9SoUQCMGjWK7dv3Lm71VsrrNLCftLbuXIAxY8bUtN5mVhqPQLb+UFbnAHcMyAzFdmprHE4GVpKRI0eydetWALZu3Upra643Ma9RXqcBM2tATgZWknPPPZclS7JxhkuWLGH69Om5Tbsoo9NAHapuZiXwFNZNKNeU8NSCD/TL63d1dbFmzRqef/552trauOqqq5g3bx4f+9jHWLRoEWPGjOFHP/pRrviLvNlp4GXg05B1GpCU6zQA+3YaMLMG42Rg+1m6dGnB+OrVqwvGI6KsTgPWXCQNA9YBv46ID0oaCywDRgC/Aj4VEa/Ws479qb+/fDUKNxOZ2YH8NfBE3vOrgWvSRJU7gTl1qZXVlJOBmRUlqQ34ANlYINLg0PcDt6Qi+RNVWhNzMjCzvnwT+HvgjfT8ncCuiNiTnvc5fsQTSzYPJwMzK0jSB4HtEfFgfrhAUY8fGQR8AdnMinkvcK6kc4C3AYeRnSkMl9SSzg48fmSQ8JmBmRUUEVdERFtEtAMzgJ9HxAXAPcD5qVj+RJUNxyO6S+dkYGbluhy4VNImsmsIi+pcH6sBNxOZ2QFFxBpgTXq8hWxachtEfGZgZmZOBmZm5mRgZmZUmQwkfV7SY5IelbRU0tskjZV0X1r39mZJB6WyB6fnm9L29rzXuSLFN0g6q7pfyczMylVxMpA0Gvg/QGdEjAeGkXU/KzZvyRxgZ0T8IXBNKoekk9J+J5Mti/idNDGWmZkNkGqbiVqA35PUAhwCbKX4vCXT03PS9slpnpPpwLKIeCUiniSbCtk9FczMBlDFySAifg18HXiaLAm8CDxI8XlL9q6Jm7a/SNZHuay1cj3XiZlZ7VXTTHQE2bf6scDRwNuBswsUzc1bUmxOE891YmZWZ9U0E/0F8GREPBcRrwG3An9Cmrcklcmft2Tvmrhp++HADrxWrplZ3VWTDJ4GJko6JLX9TwYep/i8JSvSc9L2n0dEpPiM1NtoLDAOuL+KepmZWZkqno4iIu6TdAvZsnd7gIeAhcDtwDJJ/5BiuXlLFgE/SPOZ7CDrQUREPCZpOVki2QNcHBGvV1ovs0p5UjMbyqqamygirgSu7BUuOG9JRPwv8NEirzMfmF9NXczMrHIegWxmZk4GVroNGzbQ0dGx93bYYYcBtEr6kqRfS+pOt3Ny+3h0uVlz8BTWDSDXVv3Ugg/UuSZ9O/744+nu7gbg9ddfZ/To0ezevXtX2nxNRHw9v3yv0eVHA3dLOs7XhMwaj88MrCKrV6/mXe96F8CrfRTz6HKzJuFkYBVZtmwZXV1d+aFLJK2XtDgNSIQSR5d7ZPm+2ufd7p5NNuCcDKxsr776KitWrOCjH93bOex64F1AB9nUJN9I8ZJGl3tkuVn9ORlY2e68804mTJjAyJEjAYiIbRHxekS8AXyPN5uCPLrcrEk4GVjZli5duk8TkaRReZvPAx5Njz26vIml9Unul/RwWrfkqhQvuGaJNTcnAyvLyy+/zKpVq/jwhz+cH/6apEckrQfOBD4P2ehyIDe6fCUeXd5sXgHeHxHvJmsCnCZpIsXXLLEm5q6lVpZDDjmEF154YZ9YRHyqWHmPLm9eae6wl9LTt6ZbkK1Z8okUXwJ8iey6kTUxnxmYWVGShknqBrYDq4DNFF+zpPe+7iXWRJwMzKyo1DGgg+zi/2nAiYWKFdnXvcSaiJOBmR1QROwC1gATKb5miTUxJwMzK0jSUZKGp8e/R7ag1RMUX7Ok7jxgr3K+gGxmxYwClkgaRvbFcXlE/FTS4xRes8SamJOBmRUUEeuB9xSIF1yzxJpbVc1EkoZLukXSf0p6QtIZkkZIWpUGpKzKzVOjzHVpOuP1kibkvc7MVH6jpJnFf6KZmfWHaq8ZXAusjIgTgHeTtSfOA1anASmr03OAs8lGoI4D5pL6JUsaQbZa2ulk3zauzJvozMzMBkDFyUDSYcAkUnthRLyaehxMJxuIQrr/UHo8HbgpMveS9UgYBZwFrIqIHRGxk6wv87RK62VmZuWr5szgWOA54PuSHpJ0g6S3AyMjYitAum9N5YtNZ1zSNMfgQSw2tLhnjA2kai4gtwATgM9FxH2SruXNJqFCik1nXNI0x5ANYgEWAnR2dhYsY1Yuf+CaVZcMeoCeiLgvPb+FLBlskzQqIramZqDteeULTWfcA7yvV3xNFfUysyHOCb58FTcTRcRvgGckHZ9Ck8lmp1xBNhAF9h2QsgK4MPUqmgi8mJqR7gKmSjoiXTiemmJmZjZAqh1n8Dngh2k+8y3Ap0mDUyTNAZ4Gcsth3QGcQ7YO7supLBGxQ9JXgAdSuS9HxI4q62VmZmWoKhlERDfQWWDT5AJlA7i4yOssBhZXUxczs6GgdxPYUws+UJPX9dxEZmbmZGBmZk4GZmaGk4GZmeFkYGZmOBmYmRlez8DK1N7ezqGHHsqwYcNoacnePmnm2ZuBduAp4GMRsVOSyGa2PYdsbMmsiPhVXSpuZn1yMqijZh0yf88993DkkUcCkH3e7522fIGkeen55ew7bfnpZNOWn16POptZ39xMZLVQ7rTlZtZgfGZgZZHE1KlTkcRnPvOZXHifacslHWja8q29XnMu2YJHjBkzpj+rb2ZFOBlYWX75y19y9NFHs337dqZMmQLwjj6KlzQ9uacmb0ySjgFuAn4feANYGBHXFrtGVI86NmtTayNyM5GV5eijjwagtbWV8847D+DtpGnLAUqcttyawx7gsog4EZgIXCzpJIovbWtNzMnASva73/2O3bt37338s5/9DOB/KH/acmsCEbE11/srInaTrXE+muLXiKyJuZnISrZt27bc2QB79uzhE5/4BGvXrv0tsIAypi235iOpHXgPcB/FrxH13sfXgpqIk0ETy7WX1moK2wM59thjefjhh/eJffGLXyQiXqDMacuteUh6B/CvwN9ExG9Td+ID8rWg5uJmIjMrStJbyRLBDyPi1hQudo3ImljVyUDSMEkPSfppej5W0n2SNkq6Oa2ChqSD0/NNaXt73mtckeIbJJ1VbZ3MrHppBPki4ImI+Ke8TcWuEVkTq8WZwV+TXVjKuRq4JvU02AnMSfE5wM6I+EPgmlSO1DthBnAyMA34jqRhNaiXmVXnvcCngPdL6k63c8iuEU2RtBGYkp5bk6sqGUhqAz4A3JCeC3g/cEsq0ns0aq4Hwi3A5FR+OrAsIl6JiCfJLjaeVk29zKx6EfEfEaGIOCUiOtLtjoh4ISImR8S4dO81yweBas8Mvgn8PdmAFIB3ArsiYk96nhtxCnmjUdP2F1P5YqNU9yNprqR1ktY999xzVVbdzMxyKk4Gkj4IbI+IB/PDBYrGAbaVNEoVst4JEdEZEZ1HHXVUWfU1a1bt8273SFvrd9V0LX0vcG5qQ3wbcBjZmcJwSS3p23/+iNPcaNQeSS3A4cAOPErV6sgfsmaZis8MIuKKiGiLiHayC8A/j4gLgHuA81Ox3qNRcz0Qzk/lI8VnpN5GY8mmO76/0nqZmVn5+mPQ2eXAMkn/ADxE1jWNdP8DSZvIzghmAETEY5KWA4+TzYVycUS83g/1MjOzImqSDCJiDbAmPd5Cgd5AEfG/vDlNQe9t84H5taiLmZmVzyOQzczMcxPVgy9amlmj8ZmBmZk5GZiZmZOBmZnhZGBmTcrX3mrLycCsSfjDz/qTk4GZmTkZmJmZk4GZmeFkYGZmOBkMCgM13/0zzzzDmWeeyYknnsjJJ5/MtddeC4CkL0n6da+lEXPbvL61WRPwdBQDqNl7g7S0tPCNb3yDCRMmsHv3bk499VTI1rKAbN3rr+eX77W+9dHA3ZKO86y0Zo3HZwZWslGjRjFhwgQADj30UE488USAg/rYxetbNzlJiyVtl/RoXmyEpFWSNqb7I+pZR6sNJwOryFNPPcVDDz0E8FIKXSJpffrwyH04lLS+tde2bmg3AtN6xeYBqyNiHLA6Pbcm52RgZXvppZf4yEc+wje/+U2AN4DrgXcBHcBW4BupaEnrW9djbWuvK1yaiPgF2WJU+aYDS9LjJcCHBrRS1i8qTgaSjpF0j6QnJD0m6a9TvOAppDLXpYuJ6yVNyHutman8Rkkzi/1Mq7/XXnuNj3zkI1xwwQV8+MMfBiAitkXE6xHxBvA93mwK8vrWNdYgSWxkRGwFSPethQr5jK+5VHNmsAe4LCJOBCYCF6cLhsVOIc8mW994HDCX7NskkkYAVwKnk32IXOk2yMYUEcyZM4cTTzyRSy+9dG9c0qi8YucBufZlr289hNXjjM8qV3FvovSNIPftYLekJ8jag6cD70vFlpAth3l5it8UEQHcK2l4+hB5H7AqInYASFpF1ka5tNK6Wf/45S9/yQ9+8AP+6I/+iI6Ojlz4cOBrkjrImoCeAj4DXt96ENsmaVREbE3/w9vrXSGrXk26lkpqB94D3EevU0hJuVPIYhcTS7rImH7OXLKzCsaMGVOLqlsZ/vRP/5Qsl79J0osR8ali+3h960FpBTATWJDuf1Lf6lgtVH0BWdI7gH8F/iYifttX0QKx6CO+f9CnnWYDStJSYC1wvKQeSXPIksAUSRuBKem5NbmqzgwkvZUsEfwwIm5N4WKnkMUuJvbwZrNSLr6mmnqZWW1ERFeRTZMHtCLW76rpTSRgEfBERPxT3qbcKSTsewq5Argw9SqaCLyYmpPuAqZKOiJdOJ6aYlamBuhl0vAapDeOWcOp5szgvcCngEckdafY/yU7ZVyeTiefBj6att0BnEM2CvVl4NMAEbFD0leAB1K5L+cuJg8m/gAys0ZWTW+i/6Bwez8UOIVMvYguLvJai4HFldZlKHAysaEu9z/w1IIP1Lkmg5NHIJs1ITd3Wa05GZiZmZOBmTUmn/0MLCcDMzPz4jY2NPgbplnfnAz62UB/CLnHhQ0GTt4Dz81EZk3M7epWKz4zsEHNH5RmpXEyMLOm4gS/r1o1DTsZ9BO/Yc2smTgZDFK+kGzNpK8vT/5iVZpCf6dy/v99AdlsEPCFZKuWzwxqzP+QZtaMfGYwyA3lb4xD9fduFj4+jcXJoEaG8oeuNQ6/B61STgZmg4y/mFglGuaagaRpwLXAMOCGiGiaRbZr9Y/XPu/2fuv9U6/eRQN5XP0BuK/+POa1Oq4+Zo2jIZKBpGHAt4EpQA/wgKQVEfF4fWtWWDO/gfsz4fTWbMd1sKp1UvBxHZwaIhkApwGbImILgKRlwHRgwN5czfwBX65iv2s/JIl+O65D6XjVSg2/CNT9/9Vqr1GSwWjgmbznPcDpvQtJmgvMTU9fkrShH+pyJPB8P7xuSXT1Pk8HtC69fnZvxeryB33sM5DHdUD+Vgf4G+XU9T1UwN76lFh/aJzjWitVH5My/nb9Wo9yFKlzwWPbKMlABWKxXyBiIbCwXysirYuIzv78GaUaBHUZsOM6CP5W/aYf6tMw/6+lapRj0ij1KKRRehP1AMfkPW8Dnq1TXax2fFwHJx/XQahRksEDwDhJYyUdBMwAVtS5TlY9H9fByZHg2WUAAAPJSURBVMd1EGqIZqKI2CPpEuAusq5qiyPisTpVpyFOa5OmrssAH9em/lv1s5rWp8H+X0vVKMekUeqxH0Xs19RnZmZDTKM0E5mZWR05GZiZ2dBMBpJGSFolaWO6P6JAmQ5JayU9Jmm9pI/nbbtR0pOSutOto4I6TJO0QdImSfMKbD9Y0s1p+32S2vO2XZHiGySdVe7PLrMel0p6PP0NVkv6g7xtr+f9DQb0AmK1x7BGdaj4GNZaNcdxMKr3+6OR3hsli4ghdwO+BsxLj+cBVxcocxwwLj0+GtgKDE/PbwTOr+LnDwM2A8cCBwEPAyf1KvNXwD+nxzOAm9Pjk1L5g4Gx6XWG9WM9zgQOSY8vytUjPX+pWY9hDX5+xcewH/4WVR3HwXir5/ujkd4b5dyG5JkB2dD5JenxEuBDvQtExH9FxMb0+FlgO3BUjX7+3uH8EfEqkBvOX6yOtwCTJSnFl0XEKxHxJLApvV6/1CMi7omIl9PTe8n6lDeCZj6GtdbMx7G/1PP90UjvjZIN1WQwMiK2AqT71r4KSzqNLMNvzgvPT6eW10g6uMyfX2g4/+hiZSJiD/Ai8M4S961lPfLNAe7Me/42Sesk3Stpv3+2flaLY1iNao5hrVV7HAejer4/Gum9UbKGGGfQHyTdDfx+gU1fKPN1RgE/AGZGxBspfAXwG7I3z0LgcuDL5bxsgVjvPr7FypQ0FUAN65EVlD4JdAJ/nhceExHPSjoW+LmkRyKiVh+2/X0Mq1XNMay1ao9jU2rg90cjvTdKNmiTQUT8RbFtkrZJGhURW9MbYXuRcocBtwNfjIh78157a3r4iqTvA39bZvVKGc6fK9MjqQU4HNhR4r61rAeS/oLsH+zPI+KVXDydWhMRWyStAd5D7b559+sxrIFqjmGtVXUcm1UDvz8a6b1RsqHaTLQCmJkezwR+0ruAsmH2twE3RcSPem0ble5F1hb5aJk/v5Th/Pl1PB/4eWRXm1YAM1JvhLHAOOD+Mn9+yfWQ9B7gu8C5EbE9L35ErnlM0pHAexnYKYyrOoY1UM0xrLWKj+MgVs/3RyO9N0pX7yvY9biRtc2tBjam+xEp3km2ahPAJ4HXgO68W0fa9nPgEbIk8P+Ad1RQh3OA/yL7Jv2FFPsy2T8rwNuAH5FdIL4fODZv3y+k/TYAZ1f5tzhQPe4GtuX9DVak+J+kv8HD6X5OMx3DGtWh4mPYD3+Pio7jYL3V+/3RSO+NUm+ejsLMzIZsM5GZmeVxMjAzMycDMzNzMjAzM5wMzMwMJwMzM8PJwMzMgP8Pxt0iWBeXdzwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 3 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_weights(pruned_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
